#!/usr/bin/env python
#-*- encoding=utf8 -*-
from stockin import Stockin
import os

dpi = Stockin()
filels = dpi.get_files('/home/xwtech/load_test/cdpi/','.txt')

all_idsets = set()
for filepath in filels:
    print filepath
    try:
        with open(filepath,'r') as wp:
            idsets = set()
            for line in wp:
                try:
                    idsets.add(line.strip().split('\x01')[0])
                    all_idsets.add(line.strip().split('\x01')[0])
                except Exception,e:
                    print e
            print '%d ids in %s.\n' %(len(idsets),filepath)
            print 'for now %d ids' % len(all_idsets)
    except Exception,e:
        print  'open file faild. '+e
print 'final id number:%d' % len(all_idsets)
